﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading.Tasks;

namespace WI_E2_Parser
{
    class Robots
    {
        public List<string> allowedList = new List<string>();
        public List<string> disallowedList = new List<string>();
        public double crawlDelay = 0;
        string myUserAgent = "*";

        public Robots(string url)
        {
            parseRobots(url);
        }


        public void parseRobots(string URL)
        {
            WebClient client = new WebClient();
            String content = null;
            string[] robotLines = "".Split(' ');
            try
            {
                if(!URL.Equals(""))
                { 
                     Stream stream = client.OpenRead(URL);
                     StreamReader reader = new StreamReader(stream);
                     content = reader.ReadToEnd();
                     robotLines = content.Split('\n');
                }
            }
            catch(WebException e)
            {
                
            }


            int flag = 0;
            foreach (string line in robotLines)
            {
                if (line.Length > 0 && line[0] != '#')
                {
                    if (line.Equals("User-agent: *") || line.Equals("User-agent: " + myUserAgent))
                    {
                        flag = 1;
                    }
                    else if (line.Contains("User-agent"))
                    {
                        flag = 0;
                    }

                    if (flag == 1)
                    {
                        int indexOfSlash = line.IndexOf('/') - 1;
                        if(line.Length > 4 && indexOfSlash >= 1)
                        { 
                            switch (line.Substring(0, 5))
                            {
                                case "Crawl":
                                    try
                                    {
                                        crawlDelay = Double.Parse(line.Substring(12, line.Length - 12));
                                    }
                                    catch (FormatException e)
                                    {
                                        crawlDelay = 10;
                                    }
                                    break;
                                case "Disal":
                                    try
                                    { 
                                        this.disallowedList.Add(line.Substring(indexOfSlash+1, line.Length - indexOfSlash-1));
                                        break;
                                    }
                                    catch(ArgumentOutOfRangeException e)
                                    {
                                        break;
                                    }
                                    
                                case "Allow":
                                    this.allowedList.Add(line.Substring(indexOfSlash, line.Length - indexOfSlash));
                                    break;
                                default:
                                    break;
                            }
                        }
                    }
                }
            }
        }
    }
}
